{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "afc2d638",
   "metadata": {},
   "outputs": [],
   "source": [
    "#一部电影的具体网页信息\n",
    "s='<div class=\"item\">\\\n",
    "<div class=\"pic\">\\\n",
    "<em class=\"\">1</em>\\\n",
    "<a href=\"https://movie.douban.com/subject/1292052/\">\\\n",
    "<img alt=\"肖申克的救赎\" class=\"\" src=\"https://img2.doubanio.com/view/photo/s_ratio_poster/public/p480747492.jpg\" width=\"100\"/>\\\n",
    "</a>\\\n",
    "</div>\\\n",
    "<div class=\"info\">\\\n",
    "<div class=\"hd\">\\\n",
    "<a class=\"\" href=\"https://movie.douban.com/subject/1292052/\">\\\n",
    "<span class=\"title\">肖申克的救赎</span>\\\n",
    "<span class=\"title\"> / The Shawshank Redemption</span>\\\n",
    "<span class=\"other\"> / 月黑高飞(港)  /  刺激1995(台)</span>\\\n",
    "</a>\\\n",
    "<span class=\"playable\">[可播放]</span>\\\n",
    "</div>\\\n",
    "<div class=\"bd\">\\\n",
    "<p class=\"\">\\\n",
    "                            导演: 弗兰克·德拉邦特 Frank Darabont   主演: 蒂姆·罗宾斯 Tim Robbins /...<br/>\\\n",
    "                            1994 / 美国 / 犯罪 剧情\\\n",
    "                        </p>\\\n",
    "<div class=\"star\">\\\n",
    "<span class=\"rating5-t\"></span>\\\n",
    "<span class=\"rating_num\" property=\"v:average\">9.7</span>\\\n",
    "<span content=\"10.0\" property=\"v:best\"></span>\\\n",
    "<span>2465132人评价</span>\\\n",
    "</div>\\\n",
    "<p class=\"quote\">\\\n",
    "<span class=\"inq\">希望让人自由。</span>\\\n",
    "</p>\\\n",
    "</div>\\\n",
    "</div>\\\n",
    "</div>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "995e940e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# bdpat= re.compile(r'<p class=\"\">(.*?)</p>',re.S)\n",
    "# s2=re.findall(bdpat,s)\n",
    "\n",
    "# s2=str(s2).replace('/.*<br/>(/s+)','')\n",
    "# print(s2)                 \n",
    "\n",
    "s3=re.findall(':(.*?)[\"主\"|\"/\"]',s)[0]\n",
    "print(s3)\n",
    "a=[1,2]\n",
    "print(a)\n",
    "# ratenumpat=re.compile(r'<span class=\"inq\">(.*?)</span>')\n",
    "# re.findall(ratenumpat,s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cd64a9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#git命令 \n",
    "$git add readme.txt #添加版本到缓存区\n",
    "$cat readme.txt  #获取文件内容\n",
    "$git is a distributed control version system\n",
    "\n",
    "$git commit -m 'tain jia shu ju'  #添加缓存区的版本到master分支   --并对版本说明\n",
    "$git status  #查看状态\n",
    "$ git checkout --readme.txt #撤销未提交到缓存区的修改\n",
    "$git reset HEAD readme.txt  #重置指针到之前的状态，也就是撤销缓存区的修改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78940aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "a=[1,2]\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "716f09e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "66f24d2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第1条\n",
      "第2条\n",
      "第3条\n",
      "第4条\n",
      "第5条\n",
      "第6条\n",
      "第7条\n",
      "第8条\n",
      "第9条\n",
      "第10条\n",
      "第11条\n",
      "第12条\n",
      "第13条\n",
      "第14条\n",
      "第15条\n",
      "第16条\n",
      "第17条\n",
      "第18条\n",
      "第19条\n",
      "第20条\n",
      "第21条\n",
      "第22条\n",
      "第23条\n",
      "第24条\n",
      "第25条\n",
      "第26条\n",
      "第27条\n",
      "第28条\n",
      "第29条\n",
      "第30条\n",
      "第31条\n",
      "第32条\n",
      "第33条\n",
      "第34条\n",
      "第35条\n",
      "第36条\n",
      "第37条\n",
      "第38条\n",
      "第39条\n",
      "第40条\n",
      "第41条\n",
      "第42条\n",
      "第43条\n",
      "第44条\n",
      "第45条\n",
      "第46条\n",
      "第47条\n",
      "第48条\n",
      "第49条\n",
      "第50条\n",
      "第51条\n",
      "第52条\n",
      "第53条\n",
      "第54条\n",
      "第55条\n",
      "第56条\n",
      "第57条\n",
      "第58条\n",
      "第59条\n",
      "第60条\n",
      "第61条\n",
      "第62条\n",
      "第63条\n",
      "第64条\n",
      "第65条\n",
      "第66条\n",
      "第67条\n",
      "第68条\n",
      "第69条\n",
      "第70条\n",
      "第71条\n",
      "第72条\n",
      "第73条\n",
      "第74条\n",
      "第75条\n",
      "第76条\n",
      "第77条\n",
      "第78条\n",
      "第79条\n",
      "第80条\n",
      "第81条\n",
      "第82条\n",
      "第83条\n",
      "第84条\n",
      "第85条\n",
      "第86条\n",
      "第87条\n",
      "第88条\n",
      "第89条\n",
      "第90条\n",
      "第91条\n",
      "第92条\n",
      "第93条\n",
      "第94条\n",
      "第95条\n",
      "第96条\n",
      "第97条\n",
      "第98条\n",
      "第99条\n",
      "第100条\n",
      "第101条\n",
      "第102条\n",
      "第103条\n",
      "第104条\n",
      "第105条\n",
      "第106条\n",
      "第107条\n",
      "第108条\n",
      "第109条\n",
      "第110条\n",
      "第111条\n",
      "第112条\n",
      "第113条\n",
      "第114条\n",
      "第115条\n",
      "第116条\n",
      "第117条\n",
      "第118条\n",
      "第119条\n",
      "第120条\n",
      "第121条\n",
      "第122条\n",
      "第123条\n",
      "第124条\n",
      "第125条\n",
      "第126条\n",
      "第127条\n",
      "第128条\n",
      "第129条\n",
      "第130条\n",
      "第131条\n",
      "第132条\n",
      "第133条\n",
      "第134条\n",
      "第135条\n",
      "第136条\n",
      "第137条\n",
      "第138条\n",
      "第139条\n",
      "第140条\n",
      "第141条\n",
      "第142条\n",
      "第143条\n",
      "第144条\n",
      "第145条\n",
      "第146条\n",
      "第147条\n",
      "第148条\n",
      "第149条\n",
      "第150条\n",
      "第151条\n",
      "第152条\n",
      "第153条\n",
      "第154条\n",
      "第155条\n",
      "第156条\n",
      "第157条\n",
      "第158条\n",
      "第159条\n",
      "第160条\n",
      "第161条\n",
      "第162条\n",
      "第163条\n",
      "第164条\n",
      "第165条\n",
      "第166条\n",
      "第167条\n",
      "第168条\n",
      "第169条\n",
      "第170条\n",
      "第171条\n",
      "第172条\n",
      "第173条\n",
      "第174条\n",
      "第175条\n",
      "第176条\n",
      "第177条\n",
      "第178条\n",
      "第179条\n",
      "第180条\n",
      "第181条\n",
      "第182条\n",
      "第183条\n",
      "第184条\n",
      "第185条\n",
      "第186条\n",
      "第187条\n",
      "第188条\n",
      "第189条\n",
      "第190条\n",
      "第191条\n",
      "第192条\n",
      "第193条\n",
      "第194条\n",
      "第195条\n",
      "第196条\n",
      "第197条\n",
      "第198条\n",
      "第199条\n",
      "第200条\n",
      "第201条\n",
      "第202条\n",
      "第203条\n",
      "第204条\n",
      "第205条\n",
      "第206条\n",
      "第207条\n",
      "第208条\n",
      "第209条\n",
      "第210条\n",
      "第211条\n",
      "第212条\n",
      "第213条\n",
      "第214条\n",
      "第215条\n",
      "第216条\n",
      "第217条\n",
      "第218条\n",
      "第219条\n",
      "第220条\n",
      "第221条\n",
      "第222条\n",
      "第223条\n",
      "第224条\n",
      "第225条\n",
      "第226条\n",
      "第227条\n",
      "第228条\n",
      "第229条\n",
      "第230条\n",
      "第231条\n",
      "第232条\n",
      "第233条\n",
      "第234条\n",
      "第235条\n",
      "第236条\n",
      "第237条\n",
      "第238条\n",
      "第239条\n",
      "第240条\n",
      "第241条\n",
      "第242条\n",
      "第243条\n",
      "第244条\n",
      "第245条\n",
      "第246条\n",
      "第247条\n",
      "第248条\n",
      "第249条\n",
      "第250条\n",
      "完成\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import os,re,sqlite3\n",
    "import urllib.request as req ,urllib.error\n",
    "import xlwt\n",
    "os.chdir(r'C:\\Users\\Administrator\\Desktop\\数据表')\n",
    "\n",
    "#正则提取规则\n",
    "linkpat=re.compile(r'<a href=\"(.*?)\">') #正则样式对象  # .*？懒惰搜索 \n",
    "\n",
    "picpat=re.compile(r'<img.*src=\"(.*?)\"',re.S) #扩张大小size---忽略换行符\n",
    "\n",
    "titlepat=re.compile(r'<span class=\"title\">(.*)</span>')\n",
    "\n",
    "ratepat=re.compile(r'<span class=\"rating_num\" property=\"v:average\">(.*?)</span>')\n",
    "\n",
    "\n",
    "gkpat=re.compile(r'<span class=\"inq\">(.*)</span>')\n",
    "\n",
    "ratenumpat=re.compile(r'<span>(\\d*)人评价</span>')\n",
    "\n",
    "bdpat= re.compile(r'<p class=\"\">(.*?)</p>',re.S)  #全部是根据上面的网页进行的提取\n",
    "\n",
    "#第一步：主函数\n",
    "def main():\n",
    "    baseurl='https://movie.douban.com/top250?start='\n",
    "    datalist=getdata(baseurl)\n",
    "    #保存数据\n",
    "    datapath='豆瓣电影250.xlsx'\n",
    "    savedata(datalist,datapath)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def getdata(baseurl):\n",
    "    datalist=[]\n",
    "    for i in range(10):\n",
    "        url=baseurl+str(25*i)\n",
    "        html=askurl(url)\n",
    "        #拿到网页后对其解析\n",
    "        soup=bs(html,'html.parser')\n",
    "        for item in soup.find_all('div',class_=\"item\"): #对解析后的soup进行查找。div容器下的class属性为item的标签内容\n",
    "#             print(item)  #调用主函数进行查看\n",
    "            data=[]\n",
    "            item=str(item)\n",
    "            #提取播放链接\n",
    "            link=re.findall(linkpat,item)[0]\n",
    "            data.append(link)\n",
    "            #提取图片链接\n",
    "            pic=re.findall(picpat,item)[0]\n",
    "            data.append(pic)\n",
    "            #提取标题\n",
    "            title=re.findall(titlepat,item)  #名称也不能用索引，下面会分情况\n",
    "            if len(title)==2:  #名称可能存在英文名和中文名\n",
    "                ctitle=title[0]\n",
    "                data.append(ctitle)\n",
    "                otitle=title[1].replace('/','')\n",
    "                data.append(otitle)\n",
    "            else:\n",
    "                data.append(title[0])\n",
    "                data.append('')\n",
    "                \n",
    "            #提取评分\n",
    "            rate=re.findall(ratepat,item)[0]\n",
    "            data.append(rate)\n",
    "            # 评价人数\n",
    "            ratenum=re.findall(ratenumpat,item)[0]\n",
    "            data.append(ratenum)\n",
    "            #提取概况\n",
    "            gk=re.findall(gkpat,item)   #概况可能为空不能用索引\n",
    "            if len(gk)!=0:\n",
    "                data.append(gk)\n",
    "            else:\n",
    "                data.append('')  #留空\n",
    "                \n",
    "            #其他\n",
    "            bd=re.findall(bdpat,item)[0]\n",
    "            bd=re.sub('<br(\\s+)?/>(\\s+)?',\"\",bd) #替换调 br\n",
    "            bd = re.sub('/',\" \",bd)     #替换/\n",
    "            bd = bd.strip()\n",
    "            data.append(bd.replace(' ',''))  #去空格\n",
    "            datalist.append(data)\n",
    "    return datalist\n",
    "#     print(datalist)\n",
    "\n",
    "\n",
    "        \n",
    "\n",
    "def askurl(url):\n",
    "    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0'}\n",
    "    #封装头和url\n",
    "    request=req.Request(url,headers=headers)\n",
    "    html=''\n",
    "    try:\n",
    "        response=req.urlopen(request) #根据封装信息进行打开\n",
    "        html=response.read().decode('utf8') #解码\n",
    "        html=str(html) #\n",
    "    except urllib.error.URLError as e:\n",
    "        if hasattr(e,'code'):\n",
    "            print(e.code)\n",
    "        if hasattr(e,'reason'):\n",
    "            print(e.reason)               \n",
    "    return html\n",
    "\n",
    "\n",
    "\n",
    "def savedata(datalist,datapath):\n",
    "    wb=xlwt.Workbook(encoding='utf8')\n",
    "    sht=wb.add_sheet('电影',cell_overwrite_ok=True)\n",
    "    #创建列名\n",
    "    col=['电影链接','图片链接','电影名称','外文名','评分','评价人数','概况','演员信息']\n",
    "    for i in range(len(col)):\n",
    "        sht.write(0,i,col[i])\n",
    "    for i in range(len(datalist)):  #将内部每个列表的长相加就是总长\n",
    "        data=datalist[i]\n",
    "        print('第%d条'%(i+1))\n",
    "        for j in range(len(data)):\n",
    "            sht.write(i+1,j,data[j])\n",
    "    wb.save(datapath)\n",
    "\n",
    "\n",
    "# #保存到数据库--初始化数据库\n",
    "\n",
    "# def init(dbpath):\n",
    "#     sql='''create table move(\n",
    "#     id int primary key autoincrement,\n",
    "#     into_link text,\n",
    "#     img_link text,\n",
    "#     cname varchar,\n",
    "#     oname varchar,\n",
    "#     rate numeric,\n",
    "#     ratenum numeric,\n",
    "#     info text\n",
    "#     )'''\n",
    "#     con=sqlite3.connect(dbpath)  #连接数据库，没有的话会先创建\n",
    "#     cursor=con.cursor() #创建游标\n",
    "#     cursor.execute(sql) #执行SQL 语句\n",
    "#     con.commit() #连接提交\n",
    "#     con.close()   #连接断开\n",
    "\n",
    "# #保存到数据库--插入数据\n",
    "# def savedb(datalist,dbpath):\n",
    "#     init(dbpath) #创建库\n",
    "#     con=sqlite3.connect(dbpath)  #连接\n",
    "#     cursor=con.cursor()\n",
    "#     for data in datalist:\n",
    "#         for j in range(len(data)):\n",
    "#             if j!=5 and j!=6:  #5和6是数字类型的不需要改\n",
    "#                 j=str(j)  #字符串化   全部循环完后才 串接\n",
    "#         sql='''\n",
    "#         insert into move\n",
    "#         values(%s)\n",
    "#         '''%','.join(j)  #连接成一行\n",
    "#         cursor.execute(sql)\n",
    "#         con.commit()\n",
    "#     con.close()\n",
    "\n",
    "\n",
    "if __name__=='__main__':\n",
    "    main()\n",
    "    print('完成')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "34d4539c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1', '1', 2, '5', '5', '1', '1', 2]\n"
     ]
    }
   ],
   "source": [
    "a='<span class=\"title\">末代皇帝</span>\\\n",
    "<span class=\"title\">&nbsp;/&nbsp;The Last Emperor</span>'\n",
    "titlepat=re.compile(r'<span class=\"title\">(.*?)</span>')\n",
    "b=re.findall(titlepat,a)[1]\n",
    "c=[[1,2,5],[1,2]]\n",
    "s=sum([len(i) for i in c])\n",
    "s=[str(j) for i in c for j in i]\n",
    "s\n",
    "s=[]\n",
    "for i in c:\n",
    "    for j in range(len(i)):\n",
    "        if j!=1:\n",
    "            i[j]=str(i[j])\n",
    "            s.append(i[j])\n",
    "        s.append(i[j])\n",
    "            \n",
    "print(s)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "4db169ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save....\n",
      "第1条\n",
      "第2条\n",
      "第3条\n",
      "第4条\n",
      "第5条\n",
      "第6条\n",
      "第7条\n",
      "第8条\n",
      "第9条\n",
      "第10条\n",
      "第11条\n",
      "第12条\n",
      "第13条\n",
      "第14条\n",
      "第15条\n",
      "第16条\n",
      "第17条\n",
      "第18条\n",
      "第19条\n",
      "第20条\n",
      "第21条\n",
      "第22条\n",
      "第23条\n",
      "第24条\n",
      "第25条\n",
      "第26条\n",
      "第27条\n",
      "第28条\n",
      "第29条\n",
      "第30条\n",
      "第31条\n",
      "第32条\n",
      "第33条\n",
      "第34条\n",
      "第35条\n",
      "第36条\n",
      "第37条\n",
      "第38条\n",
      "第39条\n",
      "第40条\n",
      "第41条\n",
      "第42条\n",
      "第43条\n",
      "第44条\n",
      "第45条\n",
      "第46条\n",
      "第47条\n",
      "第48条\n",
      "第49条\n",
      "第50条\n",
      "第51条\n",
      "第52条\n",
      "第53条\n",
      "第54条\n",
      "第55条\n",
      "第56条\n",
      "第57条\n",
      "第58条\n",
      "第59条\n",
      "第60条\n",
      "第61条\n",
      "第62条\n",
      "第63条\n",
      "第64条\n",
      "第65条\n",
      "第66条\n",
      "第67条\n",
      "第68条\n",
      "第69条\n",
      "第70条\n",
      "第71条\n",
      "第72条\n",
      "第73条\n",
      "第74条\n",
      "第75条\n",
      "第76条\n",
      "第77条\n",
      "第78条\n",
      "第79条\n",
      "第80条\n",
      "第81条\n",
      "第82条\n",
      "第83条\n",
      "第84条\n",
      "第85条\n",
      "第86条\n",
      "第87条\n",
      "第88条\n",
      "第89条\n",
      "第90条\n",
      "第91条\n",
      "第92条\n",
      "第93条\n",
      "第94条\n",
      "第95条\n",
      "第96条\n",
      "第97条\n",
      "第98条\n",
      "第99条\n",
      "第100条\n",
      "第101条\n",
      "第102条\n",
      "第103条\n",
      "第104条\n",
      "第105条\n",
      "第106条\n",
      "第107条\n",
      "第108条\n",
      "第109条\n",
      "第110条\n",
      "第111条\n",
      "第112条\n",
      "第113条\n",
      "第114条\n",
      "第115条\n",
      "第116条\n",
      "第117条\n",
      "第118条\n",
      "第119条\n",
      "第120条\n",
      "第121条\n",
      "第122条\n",
      "第123条\n",
      "第124条\n",
      "第125条\n",
      "第126条\n",
      "第127条\n",
      "第128条\n",
      "第129条\n",
      "第130条\n",
      "第131条\n",
      "第132条\n",
      "第133条\n",
      "第134条\n",
      "第135条\n",
      "第136条\n",
      "第137条\n",
      "第138条\n",
      "第139条\n",
      "第140条\n",
      "第141条\n",
      "第142条\n",
      "第143条\n",
      "第144条\n",
      "第145条\n",
      "第146条\n",
      "第147条\n",
      "第148条\n",
      "第149条\n",
      "第150条\n",
      "第151条\n",
      "第152条\n",
      "第153条\n",
      "第154条\n",
      "第155条\n",
      "第156条\n",
      "第157条\n",
      "第158条\n",
      "第159条\n",
      "第160条\n",
      "第161条\n",
      "第162条\n",
      "第163条\n",
      "第164条\n",
      "第165条\n",
      "第166条\n",
      "第167条\n",
      "第168条\n",
      "第169条\n",
      "第170条\n",
      "第171条\n",
      "第172条\n",
      "第173条\n",
      "第174条\n",
      "第175条\n",
      "第176条\n",
      "第177条\n",
      "第178条\n",
      "第179条\n",
      "第180条\n",
      "第181条\n",
      "第182条\n",
      "第183条\n",
      "第184条\n",
      "第185条\n",
      "第186条\n",
      "第187条\n",
      "第188条\n",
      "第189条\n",
      "第190条\n",
      "第191条\n",
      "第192条\n",
      "第193条\n",
      "第194条\n",
      "第195条\n",
      "第196条\n",
      "第197条\n",
      "第198条\n",
      "第199条\n",
      "第200条\n",
      "第201条\n",
      "第202条\n",
      "第203条\n",
      "第204条\n",
      "第205条\n",
      "第206条\n",
      "第207条\n",
      "第208条\n",
      "第209条\n",
      "第210条\n",
      "第211条\n",
      "第212条\n",
      "第213条\n",
      "第214条\n",
      "第215条\n",
      "第216条\n",
      "第217条\n",
      "第218条\n",
      "第219条\n",
      "第220条\n",
      "第221条\n",
      "第222条\n",
      "第223条\n",
      "第224条\n",
      "第225条\n",
      "第226条\n",
      "第227条\n",
      "第228条\n",
      "第229条\n",
      "第230条\n",
      "第231条\n",
      "第232条\n",
      "第233条\n",
      "第234条\n",
      "第235条\n",
      "第236条\n",
      "第237条\n",
      "第238条\n",
      "第239条\n",
      "第240条\n",
      "第241条\n",
      "第242条\n",
      "第243条\n",
      "第244条\n",
      "第245条\n",
      "第246条\n",
      "第247条\n",
      "第248条\n",
      "第249条\n",
      "第250条\n",
      "爬取完毕！\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from bs4 import BeautifulSoup     #网页解析，获取数据\n",
    "import urllib.request,urllib.error      #制定URL，获取网页数据\n",
    "import xlwt     #进行excel操作\n",
    "import sqlite3  #进行SQLite数据库操作\n",
    "import os,re\n",
    "os.chdir(r'C:\\Users\\Administrator\\Desktop\\数据表')\n",
    "\n",
    "\n",
    "def main():\n",
    "    baseurl = \"https://movie.douban.com/top250?start=\"\n",
    "    #1.爬取网页\n",
    "    datalist = getData(baseurl)\n",
    "    savepath = \"豆瓣电影Top250.xls\"\n",
    "    dbpath = \"movie.db\"\n",
    "    #3.保存数据\n",
    "    saveData(datalist,savepath)\n",
    "#     saveData2DB(datalist,dbpath)\n",
    "\n",
    "    #askURL(\"https://movie.douban.com/top250?start=\")\n",
    "\n",
    "#影片详情链接的规则\n",
    "findLink = re.compile(r'<a href=\"(.*?)\">')     #创建正则表达式对象，表示规则（字符串的模式）\n",
    "#影片图片\n",
    "findImgSrc = re.compile(r'<img.*src=\"(.*?)\"',re.S)   #re.S 让换行符包含在字符中\n",
    "#影片片名\n",
    "findTitle = re.compile(r'<span class=\"title\">(.*)</span>')\n",
    "#影片评分\n",
    "findRating = re.compile(r'<span class=\"rating_num\" property=\"v:average\">(.*)</span>')\n",
    "#找到评价人数\n",
    "findJudge = re.compile(r'<span>(\\d*)人评价</span>')\n",
    "#找到概况\n",
    "findInq = re.compile(r'<span class=\"inq\">(.*)</span>')\n",
    "#找到影片的相关内容\n",
    "findBd = re.compile(r'<p class=\"\">(.*?)</p>',re.S)\n",
    "\n",
    "\n",
    "\n",
    "#爬取网页\n",
    "def getData(baseurl):\n",
    "    datalist = []\n",
    "    for i in range(0,10):       #调用获取页面信息的函数，10次\n",
    "        url = baseurl + str(i*25)\n",
    "        html = askURL(url)      #保存获取到的网页源码\n",
    "\n",
    "         # 2.逐一解析数据\n",
    "        soup = BeautifulSoup(html,\"html.parser\")\n",
    "        for item in soup.find_all('div',class_=\"item\"):     #查找符合要求的字符串，形成列表\n",
    "            #print(item)   #测试：查看电影item全部信息\n",
    "            data = []    #保存一部电影的所有信息\n",
    "            item = str(item)\n",
    "\n",
    "            #影片详情的链接\n",
    "            link = re.findall(findLink,item)[0]     #re库用来通过正则表达式查找指定的字符串\n",
    "            data.append(link)                       #添加链接\n",
    "\n",
    "            imgSrc = re.findall(findImgSrc,item)[0]\n",
    "            data.append(imgSrc)                     #添加图片\n",
    "\n",
    "            titles = re.findall(findTitle,item)     #片名可能只有一个中文名，没有外国名\n",
    "            if(len(titles) == 2):\n",
    "                ctitle = titles[0]                  #添加中文名\n",
    "                data.append(ctitle)\n",
    "                otitle = titles[1].replace(\"/\",\"\")  #去掉无关的符号\n",
    "                data.append(otitle)                 #添加外国名\n",
    "            else:\n",
    "                data.append(titles[0])\n",
    "                data.append(' ')        #外国名字留空\n",
    "\n",
    "            rating = re.findall(findRating,item)[0]\n",
    "            data.append(rating)                        #添加评分\n",
    "\n",
    "            judgeNum = re.findall(findJudge,item)[0]\n",
    "            data.append(judgeNum)                       #提加评价人数\n",
    "\n",
    "            inq = re.findall(findInq,item)\n",
    "            if len(inq) != 0:\n",
    "                inq = inq[0].replace(\"。\",\"\")    #去掉句号\n",
    "                data.append(inq)                # 添加概述\n",
    "            else:\n",
    "                data.append(\" \")                #留空\n",
    "\n",
    "            bd = re.findall(findBd,item)[0]\n",
    "            bd = re.sub('<br(\\s+)?/>(\\s+)?',\" \",bd)   #去掉<br/>\n",
    "            bd = re.sub('/',\" \",bd)     #替换/\n",
    "            data.append(bd.strip())     #去掉前后的空格\n",
    "\n",
    "            datalist.append(data)       #把处理好的一部电影信息放入datalist\n",
    "\n",
    "    return datalist\n",
    "\n",
    "\n",
    "\n",
    "#得到指定一个URL的网页内容\n",
    "def askURL(url):\n",
    "    head = {                #模拟浏览器头部信息，向豆瓣服务器发送消息\n",
    "        \"User-Agent\": \"Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122  Safari / 537.36\"\n",
    "    }\n",
    "                            #用户代理，表示告诉豆瓣服务器，我们是什么类型的机器、浏览器（本质上是告诉浏览器，我们可以接收什么水平的文件内容）\n",
    "\n",
    "    request = urllib.request.Request(url,headers=head)\n",
    "    html = \"\"\n",
    "    try:\n",
    "        response = urllib.request.urlopen(request)\n",
    "        html = response.read().decode(\"utf-8\")\n",
    "        #print(html)\n",
    "    except urllib.error.URLError as e:\n",
    "        if hasattr(e,\"code\"):\n",
    "            print(e.code)\n",
    "        if hasattr(e,\"reason\"):\n",
    "            print(e.reason)\n",
    "    return html\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#保存数据\n",
    "def saveData(datalist,savepath):\n",
    "    print(\"save....\")\n",
    "    book = xlwt.Workbook(encoding=\"utf-8\",style_compression=0)  #创建workbook对象\n",
    "    sheet = book.add_sheet('豆瓣电影Top250',cell_overwrite_ok=True)    #创建工作表\n",
    "    col = (\"电影详情链接\",\"图片链接\",\"影片中文名\",\"影片外国名\",\"评分\",\"评价数\",\"概况\",\"相关信息\")\n",
    "    for i in range(0,8):\n",
    "        sheet.write(0,i,col[i]) #列名\n",
    "    for i in range(0,250):\n",
    "        print(\"第%d条\" %(i+1))\n",
    "        data = datalist[i]\n",
    "        for j in range(0,8):\n",
    "            sheet.write(i+1,j,data[j])      #数据\n",
    "\n",
    "    book.save(savepath)       #保存\n",
    "\n",
    "\n",
    "def saveData2DB(datalist,dbpath):\n",
    "    init_db(dbpath)\n",
    "    conn = sqlite3.connect(dbpath)\n",
    "    cur = conn.cursor()\n",
    "\n",
    "    for data in datalist:\n",
    "        for index in range(len(data)):\n",
    "            if index == 4 or index == 5:\n",
    "                continue\n",
    "            data[index] = '\"'+data[index]+'\"'\n",
    "        sql = '''\n",
    "                insert into movie250 (\n",
    "                info_link,pic_link,cname,ename,score,rated,instroduction,info) \n",
    "                values(%s)'''%\",\".join(data)\n",
    "        print(sql)\n",
    "        cur.execute(sql)\n",
    "        conn.commit()\n",
    "    cur.close()\n",
    "    conn.close()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def init_db(dbpath):\n",
    "    sql = '''\n",
    "        create table movie250 \n",
    "        (\n",
    "        id integer primary key autoincrement,\n",
    "        info_link text,\n",
    "        pic_link text,\n",
    "        cname varchar,\n",
    "        ename varchar,\n",
    "        score numeric ,\n",
    "        rated numeric ,\n",
    "        instroduction text,\n",
    "        info text\n",
    "        )\n",
    "    \n",
    "    '''  #创建数据表\n",
    "    conn = sqlite3.connect(dbpath)\n",
    "    cursor = conn.cursor()\n",
    "    cursor.execute(sql)\n",
    "    conn.commit()\n",
    "    conn.close()\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":          #当程序执行时\n",
    "#调用函数\n",
    "    main()\n",
    "    #init_db(\"movietest.db\")\n",
    "    print(\"爬取完毕！\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
